{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-22T11:35:07.525791Z",
     "start_time": "2020-09-22T11:35:07.522612Z"
    }
   },
   "outputs": [],
   "source": [
    "#!sudo pip3 install \"foolbox==1.8.0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-22T11:35:08.491505Z",
     "start_time": "2020-09-22T11:35:07.529546Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.4.0+cu100\n",
      "0.5.0+cu100\n",
      "1.8.0\n",
      "1.19.0\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import warnings\n",
    "import torch\n",
    "import torchvision\n",
    "import foolbox\n",
    "import numpy as np\n",
    "print(torch.__version__)\n",
    "print(torchvision.__version__)\n",
    "print(foolbox.__version__)\n",
    "print(np.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-22T11:35:08.925632Z",
     "start_time": "2020-09-22T11:35:08.494985Z"
    }
   },
   "outputs": [],
   "source": [
    "model = torchvision.models.resnet18(pretrained=True).eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-22T11:35:11.738032Z",
     "start_time": "2020-09-22T11:35:08.927859Z"
    }
   },
   "outputs": [],
   "source": [
    "preprocessing = (np.asarray([0.485, 0.456, 0.406]).reshape((3, 1, 1)), np.asarray([0.229, 0.224, 0.225]).reshape((3, 1, 1)))\n",
    "fmodel = foolbox.models.PyTorchModel(model, bounds=(0, 1), num_classes=1000, preprocessing=preprocessing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-22T11:35:11.755412Z",
     "start_time": "2020-09-22T11:35:11.741660Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(16, 3, 224, 224) (16,)\n"
     ]
    }
   ],
   "source": [
    "images = np.load(\"images.npy\")\n",
    "labels = np.load(\"labels.npy\")\n",
    "print(images.shape, labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-22T11:35:11.763303Z",
     "start_time": "2020-09-22T11:35:11.757842Z"
    }
   },
   "outputs": [],
   "source": [
    "def accuracy(fmodel, images, labels):\n",
    "    assert len(images) == len(labels)\n",
    "    classes = np.asarray([fmodel.predictions(images[i]).argmax(axis=0) for i in range(len(images))])\n",
    "    return (classes == labels).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-22T11:35:11.861800Z",
     "start_time": "2020-09-22T11:35:11.766791Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9375\n"
     ]
    }
   ],
   "source": [
    "print(accuracy(fmodel, images, labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-22T11:35:19.045631Z",
     "start_time": "2020-09-22T11:35:11.863870Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "88.9 ms ± 8.24 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit accuracy(fmodel, images, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-22T11:35:23.168698Z",
     "start_time": "2020-09-22T11:35:19.047958Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.02 ms ± 338 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit accuracy(fmodel, images[:1], labels[:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-22T11:35:23.177998Z",
     "start_time": "2020-09-22T11:35:23.171378Z"
    }
   },
   "outputs": [],
   "source": [
    "def run_attack(attack, fmodel, images, labels, epsilons):\n",
    "    assert len(images) == len(labels)\n",
    "    success = []\n",
    "    with warnings.catch_warnings():\n",
    "        warnings.simplefilter(\"ignore\")\n",
    "        for i in range(len(images)):\n",
    "            adv = attack(images[i], label=labels[i], unpack=False)\n",
    "            success.append([adv.distance.value <= eps for eps in epsilons])\n",
    "    success = np.asarray(success).T\n",
    "    return success"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-22T11:38:04.962813Z",
     "start_time": "2020-09-22T11:35:23.180342Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "attack took 161.8 seconds\n"
     ]
    }
   ],
   "source": [
    "# apply the attack\n",
    "start = time.time()\n",
    "attack = foolbox.attacks.PGD(fmodel, distance=foolbox.distances.Linf)\n",
    "epsilons = [0.002]\n",
    "success = run_attack(attack, fmodel, images, labels, epsilons=epsilons)\n",
    "print(f\"attack took {time.time() - start:.1f} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-22T11:38:04.972084Z",
     "start_time": "2020-09-22T11:38:04.965966Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.002 0.0625\n"
     ]
    }
   ],
   "source": [
    "# calculate and report the robust accuracy\n",
    "robust_accuracy = 1 - success.mean(axis=-1)\n",
    "for eps, acc in zip(epsilons, robust_accuracy):\n",
    "    print(eps, acc.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-22T11:40:49.550952Z",
     "start_time": "2020-09-22T11:38:04.974466Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "attack took 164.6 seconds\n"
     ]
    }
   ],
   "source": [
    "# apply the attack\n",
    "start = time.time()\n",
    "attack = foolbox.attacks.PGD(fmodel, distance=foolbox.distances.Linf)\n",
    "epsilons = [0.0, 0.001, 0.01, 0.03, 0.1, 0.3, 0.5, 1.0]\n",
    "success = run_attack(attack, fmodel, images, labels, epsilons=epsilons)\n",
    "print(f\"attack took {time.time() - start:.1f} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-22T11:40:49.560523Z",
     "start_time": "2020-09-22T11:40:49.553662Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 0.9375\n",
      "0.001 0.375\n",
      "0.01 0.0\n",
      "0.03 0.0\n",
      "0.1 0.0\n",
      "0.3 0.0\n",
      "0.5 0.0\n",
      "1.0 0.0\n"
     ]
    }
   ],
   "source": [
    "# calculate and report the robust accuracy\n",
    "robust_accuracy = 1 - success.mean(axis=-1)\n",
    "for eps, acc in zip(epsilons, robust_accuracy):\n",
    "    print(eps, acc.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
